package com.chance.cc.crawler.prod.command.job.domain.news.zhihu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerJob;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.meta.core.bean.CrawlerMetaConstant;
import com.chance.cc.crawler.meta.core.bean.job.CrawlerScheduleJob;
import com.chance.cc.crawler.prod.command.job.domain.news.NewsCommonScript;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author lt
 * @version 1.0
 * @date 2021-05-20 15:47:41
 * @email okprog@sina.com
 */
public class ZHSearchSimpleCrawlerSchedulerJob extends NewsCommonScript {

    public static final String domain = "zhihu";

    public static void main(String[] args) {
        publishCrawlerScheduleJobInfo();
    }

    public static CrawlerJob publishCrawlerScheduleJobInfo(){


        CrawlerJob jdCrawlerJob = crawlerJob();

        //发布定时采集作业
        CrawlerScheduleJob crawlerScheduleJob = new CrawlerScheduleJob();
        crawlerScheduleJob.setDomain(domain);
        crawlerScheduleJob.setCrawlerJob(JSON.toJSONString(jdCrawlerJob));
        crawlerScheduleJob.setJobType(CrawlerMetaConstant.ScheduleCrawlerJobType.crawler.enumVal());
        crawlerScheduleJob.setNote("知乎搜索关键词临时采集");
        crawlerScheduleJob.setCrawlerKey(jdCrawlerJob.generateCrawlerKey());
        HttpPage page = metaServiceCommand.addOrUpdateCrawlerScheduleJob(crawlerScheduleJob);
        System.out.println("发布作业：" + page.getRawText());
        return jdCrawlerJob;

    }

    /**
     * 17765187595
     * chancedata123
     *
     * taobao采集
     * @return
     */
    public static CrawlerJob crawlerJob() {

        FilterInfo filterInfo = new FilterInfo();
        filterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        filterInfo.setCountTotalNum(100);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://www.zhihu.com/api/v4/search_v3")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .httpHead("cookie","SESSIONID=sY17U4kpziehprc308yu8qcYgLBv8hTjVGEWpiin0CX; JOID=UlwRAEjpRur9H_Z_Z-rBMXbb1W1wzmXO3j_RXEPJYcnZPNZYRPbgdZ4Y8nhgBrp6aEUFR-ZLaDCoL1yHQRJSp3E=; osd=W1sUAkLgQe__Ff94YujLOHHe12d5yWDM1DbWWUHDaM7cPtxRQ_Pif5cf93pqD71_ak8MQONJYjmvKl6NSBVXpXs=; _zap=c3e45db7-96fe-4a0b-acfa-bd5aa39edde6; _xsrf=5bf6cc73-7b63-4857-add0-2f96a0d15258; d_c0=\"AFAdY8EgFROPTgnCch2SduPSYyAXAuM9SIg=|1620615387\"; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1620615390; gdxidpyhxdE=SCw2aaTE0K4zxrDecinnp9l4f4g8SyPkP4XksmvK8p6wR85g1xSs0LSANG0InSTeR9NnQUMDaMLj%2BeQI83BoBZm6I5e%2FceWAt0%2F%2FS5vYdlguPsyx%2B08Zk%5C8%2B5Vhp8QrVx5jgADTNKyROvJqiq8hh33NnxJhgLmWBCct6Tutk9xZ%2BZjhl%3A1620616292059; _9755xjdesxxd_=32; YD00517437729195%3AWM_NI=2rxgjk6fVutwCWpGv%2BRbIoen4ejPgWs6mozFuwSoFHufO8pjh8M9LEy2WtLjJAuMdm2qiYCJ8MgkMaj%2Fvu0cLqLj1jsC8aZzviE7rL3%2BbELus1E2HvXGJHo6ZIxXGoemM2I%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eeb3cf7b82bff9aeed5b96968ab3c44b828b9fbaf561f1bca5a4ed4eb49696d1b42af0fea7c3b92af2aeb888e961f6ba9a96f77d8189c0a8f52192be8790e866f8bba6b1e65db1eaa390f94eb7a99d90c544b7ba81d9f362ace883dacf4fbbee9995cf628a9fbbaee767f2afa7bac242fc93a98cd06baaf1ba96b36990af848bb462b19df882d972b6b78bd8f148f1f5aea6cf7fb1ef8bd3cb7e9c9986a9ee3e9a97b8b9c665a7b383d3cc37e2a3; YD00517437729195%3AWM_TID=SfFPbI8BdmVFERQAEAIqljJxG3R5INhg; captcha_session_v2=\"2|1:0|10:1620615390|18:captcha_session_v2|88:RlFLeUQ3MlYzTkplRVB2aGxTcTZpbEFvdzJMYjNrbUV1czJySnVURTFyMUpaV3Iremg4VkY2MDRzZHZ4YWM5Qw==|61e840687dee3064799124501c1a097f6ae7007b2ea667127ffe51c71ff9f5e3\"; captcha_ticket_v2=\"2|1:0|10:1620615416|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfSzZXYUcuTHE4RFZsTWdDZEVFelVMMWVfVXV1amN2MWpFUHNWN24weWU0NEpseTJLc3lhMXBhT25MZGZSWmV1TTR3bFQ2ZEY5MUdfUnJhZTBlUHFvR0Utc2JPSmlra2hxVW40UTZOdnFMWURqc1YueDVxQnVLR0hkdzhtTl9KN3FZOWNrLnMwSlF6VHdCU0JzanNSQ3JUcFpqZEQ0ZGk5eEtTX1VIU3JaUTRZandhN242Rzg5VVRYOHdaYzBxb1JVNVNtNkN6Q1hhRkxlMTdwb085UzV0Sm1rMUt1VERaYy5rVHJUYThCbWhBRUVhMGwyQmhXYXFpZy1NRmlId0lUX2dVLnVxNWxfZ0IwWEQ4R1pKUEdRVEUxMDctYkVlcWxKMVQ1X2RJeDFSMTJPTWNFeEgyMF9KU3JlUmF4ZnZWTEd3MVFIblNzeGVnejVDczlFMnE3aUJPTlk5Ylpkb3R5MXJMRmxHNUZUcVFsTElHRTdOZTRNYl9jZG1SaC5rcnBYanJrbHVBOVo2X0l1WG43UC1rUEM4ZExQbi5jQnJ4UXppdWItdGNuLXNOb3VpbjR4LWxUS1lTLlVLam8tcUEuSnNoalllLUJBWGswMDc1X3hrYzRaT1RCb3dJX2lYRjBQVHV5UG5TZGRlbS5IUVhKd0xwQnFWOTJaSU5YMyJ9|c898a72512b1bf08dd8b91dd14d05ec33dd4726c9f80713448a102835dcfd79d\"; z_c0=\"2|1:0|10:1620615417|4:z_c0|92:Mi4xSnpwYklBQUFBQUFBVUIxandTQVZFeVlBQUFCZ0FsVk4tZTZGWVFBN2xIaVpRR0tEc3dacTRka0xNUFk3VjI2eDh3|91305f58d84d5345cd61322b49bb2109ba67fe033b08702f16f61ba5beabf1b6\"; tst=r; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1620615452; KLBRSID=fe78dd346df712f9c4f126150949b853|1620615451|1620615387")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .turnPageFilterInfo(filterInfo)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 7,null))
                .addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-zhihu-search-article-queue"))
                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("search");
        requestRecord.tagsCreator().bizTags().addDomain(domain);

        CrawlerRecord articleFilterCrawlerRecord = new CrawlerRecord();
        articleFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 7,null));
        articleFilterCrawlerRecord.addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-zhihu-search-answer-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("article_record_filter_info", JSON.toJSONString(articleFilterCrawlerRecord));

        CrawlerRecord commentFilterCrawlerRecord = new CrawlerRecord();
        commentFilterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24* 2 ,null));
        commentFilterCrawlerRecord.addFilterInfo(FilterUtils.redisFilterKeyInfo("filter-zhihu-search-comment-queue"));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("time_zone", "a_week");
        requestRecord.tagsCreator().bizTags().addCustomKV("scheduler_type", "simple");

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domain,"search","simple","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("zhihu_news",turnPageItem)
                .httpUrl(metaServiceHttpPrefix+"/v1/meta/zhihu/keys?site=search")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        return CrawlerJob.builder()
                .crawlerJobThreadNumber(20)
                .triggerInfo(
                        domain,
                        CrawlerMetaConstant.ScheduleJobTrigger_Cron,
                        System.currentTimeMillis(),
                        StringUtils.joinWith("-","search","simple",CrawlerMetaConstant.ScheduleJobTriggerJob_Realtime))
                .crawlerRequestQueue(CrawlerMetaConstant.redisRequestQueue(requestQueueName))
                .kafkaResultPipeline(null,kafkaTopic,null)
                .fileResultPipeline("","/data/chance_crawler_runner/logs/node/zhihu-simple.log",false)
                .requestRecord(requestRecord)
                .supportRecord(initNewsCrawlerRecord)
                .build();
    }


}
